import java.net.*; 
import java.io.*;
import java.util.ArrayList;
import javax.swing.text.*;
import javax.swing.text.html.parser.ParserDelegator;
import javax.swing.text.html.*;

public class Grabber
{
    private InputStream openInputStream(String url_address)
    {
        try
        {
            URL url = new URL(url_address);
            URLConnection connection = url.openConnection();
            InputStream instream = connection.getInputStream();
            return instream;
        }
        catch(IOException ex)
        {
            System.out.print("\n Can not open connection: " + ex);
            return null;
        }
    }
    private void closeConnection(InputStream instream)
    {
        try
        {
            instream.close();
        }
        catch(IOException ex)
        {
            System.err.print("\n Can not close input stream:" + ex);
        }
    }
    private HTMLDocument getHTMLDocument(InputStream instream)
    {
        try 
        {            
            BufferedReader bufreader = new BufferedReader(new InputStreamReader(instream));
            HTMLEditorKit htmlKit = new HTMLEditorKit();
            HTMLDocument htmlDoc = (HTMLDocument) htmlKit.createDefaultDocument();
            HTMLEditorKit.Parser parser = new ParserDelegator();
            HTMLEditorKit.ParserCallback callback = htmlDoc.getReader(0);
            parser.parse(bufreader, callback, true);
            return htmlDoc;
        }
        catch (IOException ex) 
        {
            System.err.print("\n Can not parse html document: " + ex);
            return null;
        } 
    }
    public String getTXTFile(String url_address)
    {
        InputStream instream = openInputStream(url_address);      
        String text = "";
        if(instream != null)
        {
            try 
            {
                HTMLDocument htmlDoc = getHTMLDocument(instream);
                HTMLDocument.Iterator iterator;               
                for (iterator = htmlDoc.getIterator(HTML.Tag.CONTENT); iterator.isValid(); iterator.next()) 
                {
                    int startOffset = iterator.getStartOffset();
                    int endOffset = iterator.getEndOffset();
                    int length = endOffset - startOffset;
                    text += htmlDoc.getText(startOffset, length);
                }
                return text;
            }           
            catch (BadLocationException ex) 
            {
                System.out.print("\n BadLocationException " + ex);
                return text;
            }  
            finally
            {
                closeConnection(instream);
            }
        }
        return text;
    }
    public String getContentHTMLFile(String url_address)
    {
        InputStream instream = openInputStream(url_address);
        BufferedReader bufreader = new BufferedReader(new InputStreamReader(instream));
        String text = "";
        String part_string;
        try
        {                 
            while((part_string = bufreader.readLine()) != null)
            {
                text += part_string;
            }
            return text;
        }
        catch(IOException ex)
        {
            System.err.println("Can not read string from input stream: " + ex); 
            return text;
        } 
        finally
        {
            closeConnection(instream);
        }
    }
    public ArrayList<String> getTextUrlLinks(String start_page, String url_address) 
    {
        InputStream instream = openInputStream(url_address);
        ArrayList url_list = new ArrayList();
        if(instream!=null)
        {
            HTMLDocument htmlDoc = getHTMLDocument(instream);        
            HTMLDocument.Iterator iterator;      
            RegularExpression regex = new RegularExpression();
            for (iterator = htmlDoc.getIterator(HTML.Tag.A); iterator.isValid(); iterator.next()) 
            {
                AttributeSet attributes = iterator.getAttributes();       
                String url = "" + (String)attributes.getAttribute(HTML.Attribute.HREF);        
                if(regex.isInternalUrlAddress(url))
                {
                    url_list.add(start_page + url);
                }
                if(regex.isCorrectUrlAddress(url))
                {
                    url_list.add(url);
                }
            }
        }
        return url_list;
    }
    public ArrayList<String> getImgUrlLinks(String url_address, String type)
    {
        InputStream instream = openInputStream(url_address);
        HTMLDocument htmlDoc = getHTMLDocument(instream);
        ArrayList<String> img_url_list = new ArrayList();
        HTMLDocument.Iterator iterator;      
        RegularExpression regex = new RegularExpression();
        for (iterator = htmlDoc.getIterator(HTML.Tag.IMG); iterator.isValid(); iterator.next()) 
        {
            AttributeSet attributes = iterator.getAttributes();       
            String url_img = "" + (String) attributes.getAttribute(HTML.Attribute.SRC);
            if(regex.isCorrectImgUrlAddress(url_img, type))
            {
                img_url_list.add(url_img);     
            }
        }
        return img_url_list; 
      }
    
} 


